/**@author: Otis Gospodnetic, Scott Sanner (ssanner@gmail.com) **/

/**          This class based on freely available code found at
 **          http://www.onjava.com/pub/a/onjava/2003/01/15/lucene.html?page=2
 **/
package mahmoud.utils;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LetterTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.PorterStemFilter;
//import org.apache.lucene.analysis.snowball.SnowballAnalyzer;

import java.io.Reader;
import java.util.Set;

/**
 * PorterStemAnalyzer processes input
 * text by stemming English words to their roots.
 * This Analyzer also converts the input to lower case
 * and removes stop words.  A small set of default stop
 * words is defined in the STOP_WORDS
 * array, but a caller can specify an alternative set
 * of stop words by calling non-default constructor.
 */
public class PorterStemAnalyzer extends Analyzer
{
    private static Set<Object> _stopTable;
    private boolean _bDoLowerCase;
    private boolean _bDoStem;

    /**
     * An array containing some common English words
     * that are usually not useful for searching.
     */
    public static final String[] STOP_WORDS =
    {
    	"defence",   	"state",    	"member",    	"amount",    	"period",    	"subsection",
    	"act",    	"section",    	"shall",    	"person",    	"australia",    	"australian",
    	"agreement",    	"minister",    	"commission",    	"sect",    	"paragraph",    	"am",
    	"st",    	"the",    	"of"  ,  	    	"and",    	"to",    	"a",    	"in",    	"that",
    	"is",    	"was",    	"He",    	"for",    	"it",    	"with",    	"as",   	"his",    	"on",
    	"be",    	"at",    	"by",    	"I",    	"this",    	"had",    	"not",    	"are",    	"but",
    	"from",    	"or",    	"have",    	"an",    	"they",    	"which",    	"one",    	"you",    	"were",
    	"her",    	"all",    	"She",    	"there",    	"would",    	"their",    	"we",    	"him",
    	"been",    	"has",    	"when",    	"who",    	"will",    	"more",    	"no",    	"If",    	"out",    	"so",
    	"said",    	"what",    	"up",    	"its",    	"its",    	"about",    	"into",    	"than",    	"them",
    	"can",    	"Only",    	"other",    	"new",    	"some",    	"could",    	"these",    	"two",    	"may",
    	"then",    	"do",    	"first",    	"any",    	"fr",    	"gbr",    	"de",    	"sp",   	    	"en",    	"number",
    	"my",    	"now",    	"such",    	"like",    	"our",    	"over",    	"man",    	"me",    	"even",    	"most",    	"after",
    	"also",    	"did",    	"many",    	"before",    	"must",    	"through",    	"back",    	"where",    	"much",    	"your",
    	"way",    	"well",    	"down",    	"should",    	"because",    	"each",    	"just",    	"those",    	"people",
    	"Mr",    	"how",    	"too",    	"little",    	"good",    	"very",    	"make",    	"still",    	"see",    	"own",    	"long",
    	"here",    	"GET",    	"both",    	"between",    	"being",    	"under",    	"never",    	"day",    	"same",    	"Another",
    	"know",    	"While",    	"last",    	"might",    	"great",    	"old",    	"off",    	"come",    	"since",    	"go",
    	 "against",    	"came",    	"right",    	"used",    	"take",    	"three",    	"himself",    	"few",    	"use",
    	"During",     	    	"WITHOUT",    	"again",    	"place",    	"around",    	"However",    	"small",    	"found",
    	"Mrs",    	"thought",    	"went",    	"say",    	"part",    	"once",    	"upon",    	"every",
    	"dont",    	"does",    	"got",    	"left" ,    	"Until",    	"always",    	"away",    	"something",    	"fact",
    	"though",    	"less",    	"put",    	"think",    	"almost",    	"enough",    	"far",    	"took",    	"yet",
    	"set",    	"better",    	"told",    	"Nothing",    	"end",    	"why",    	"didnt",    	"called",    	"close",    	"RSS",
    	"font",    	"next",    	"comment",    	"comments",    	"link",    	"links",    	"site",    	"sites",    	"push",    	"pull",
    	"terms",    	"lot",    	"please",    	"four",    	"five",    	"six",    	"seven",    	"eight",    	"nine",
    	"ten",    	"eleven",    	"twelve",    	"thirteen",    	"fourteen",    	"fifteen",    	"sixteen",    	"seventeen",    	"eighteen",
    	"nineteen",    	"twenty",    	"thirty",    	"forty",    	"fifty",    	"sixty",    	"seventy",    	"eighty",
    	"ninety",    	"hundred",    	"thousand",    	"million",    	"billion",    	"trillion",    	"monday",    	"tuesday",
    	"wednesday",     	"thursday",    	"friday",    	"saturday",    	"sunday",    	"weekday",    	"mon",    	"tues",    	"tue",
    	"wed",    	"thur",    	"thurs",    	"fri",    	"sat",    	"yes",    	"ps",    	"pdf", "shall", "which", "other","under",   "section", "subsection",
    	};

    /**
     * Builds an analyzer.
     */
    public PorterStemAnalyzer()
    {
        this(STOP_WORDS, true, true);
    }

    public PorterStemAnalyzer(String[] stopWords)
    {
    	this(stopWords, true, true);
    }
    
    /**
     * Builds an analyzer with the given stop words.
     *
     * @param stopWords a String array of stop words
     */
    public PorterStemAnalyzer(String[] stopWords, boolean do_lower_case, boolean do_stem)
    {
        _stopTable = StopFilter.makeStopSet(stopWords);
        _bDoLowerCase = do_lower_case;
        _bDoStem = do_stem;
    }

    /**
     * Processes the input by first converting it to
     * lower case, then by eliminating stop words, and
     * finally by performing Porter stemming on it.
     *
     * @param reader the Reader that
     *               provides access to the input text
     * @return an instance of TokenStream
     */
    public final TokenStream tokenStream(Reader reader)
    {
    	return tokenStream(null, reader);
    }

	@Override
	// We can ignore field
	public TokenStream tokenStream(String field, Reader arg1) {
		if (_bDoLowerCase) {
			TokenFilter base = new StopFilter(_bDoLowerCase, new LowerCaseTokenizer(arg1),_stopTable);
			if (_bDoStem)
				return new PorterStemFilter(base);
			else 
				return base;
		} else {
			TokenFilter base = new StopFilter(_bDoLowerCase, new LetterTokenizer(arg1),_stopTable);
			if (_bDoStem)
				return new PorterStemFilter(base);
			else 
				return base;
		}
			
	}
}